<!DOCTYPE html>
<html lang="zh">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 5.3.0">


  <link rel="apple-touch-icon" sizes="180x180" href="/yuwanzi.io/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/yuwanzi.io/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/yuwanzi.io/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/yuwanzi.io/images/logo.svg" color="#222">

<link rel="stylesheet" href="/yuwanzi.io/css/main.css">



<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@5.15.1/css/all.min.css">
  <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/animate.css@3.1.1/animate.min.css">

<script class="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"suyuhuan.gitee.io","root":"/yuwanzi.io/","images":"/yuwanzi.io/images","scheme":"Muse","version":"8.2.0","exturl":false,"sidebar":{"position":"left","display":"post","padding":18,"offset":12},"copycode":false,"bookmark":{"enable":false,"color":"#222","save":"auto"},"fancybox":false,"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"motion":{"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"fadeInDown","post_body":"fadeInDown","coll_header":"fadeInLeft","sidebar":"fadeInUp"}},"prism":false,"i18n":{"placeholder":"Suche...","empty":"We didn't find any results for the search: ${query}","hits_time":"${hits} results found in ${time} ms","hits":"${hits} results found"}};
  </script>
<meta name="description" content="Hadoop概述Hadoop是一个由Apache基金会所开发的分布式系统基础架构。  &amp;nbsp;&amp;nbsp;Hadoop 由许多元素构成。其最底部是 Hadoop Distributed File System（HDFS），它存储 Hadoop 集群中所有存储节点上的文件。HDFS的上一层是MapReduce 引擎，该引擎由 JobTrackers 和 TaskTrackers 组成。 HDF">
<meta property="og:type" content="article">
<meta property="og:title" content="Hadoop学习笔记(1)-HDFS">
<meta property="og:url" content="https://suyuhuan.gitee.io/yuwanzi.io/2016/07/12/2016-07-12-Hadoop-HDFS/index.html">
<meta property="og:site_name" content="玉丸子 | Blog">
<meta property="og:description" content="Hadoop概述Hadoop是一个由Apache基金会所开发的分布式系统基础架构。  &amp;nbsp;&amp;nbsp;Hadoop 由许多元素构成。其最底部是 Hadoop Distributed File System（HDFS），它存储 Hadoop 集群中所有存储节点上的文件。HDFS的上一层是MapReduce 引擎，该引擎由 JobTrackers 和 TaskTrackers 组成。 HDF">
<meta property="og:locale">
<meta property="og:image" content="http://ww4.sinaimg.cn/mw690/63503acbjw1f5w446w8pcj20bp08rweu.jpg">
<meta property="og:image" content="http://ww4.sinaimg.cn/mw690/63503acbjw1f5w4478njsj20as05wwep.jpg">
<meta property="og:image" content="http://ww2.sinaimg.cn/mw690/63503acbjw1f5w447vawzj20hq0aptar.jpg">
<meta property="og:image" content="http://ww3.sinaimg.cn/mw690/63503acbjw1f5w48p90chj20k20bzdgr.jpg">
<meta property="og:image" content="http://ww3.sinaimg.cn/mw690/63503acbjw1f5w448fsbnj20gn0a6acc.jpg">
<meta property="og:image" content="http://ww4.sinaimg.cn/mw690/63503acbjw1f5w48ovukqj20jm0ermy3.jpg">
<meta property="og:image" content="http://ww2.sinaimg.cn/mw690/63503acbjw1f5w4486azhj20g209adi1.jpg">
<meta property="article:published_time" content="2016-07-12T10:00:00.000Z">
<meta property="article:modified_time" content="2020-11-07T00:58:17.000Z">
<meta property="article:author" content="玉丸子">
<meta property="article:tag" content="Hadoop">
<meta property="article:tag" content="大数据">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="http://ww4.sinaimg.cn/mw690/63503acbjw1f5w446w8pcj20bp08rweu.jpg">


<link rel="canonical" href="https://suyuhuan.gitee.io/yuwanzi.io/2016/07/12/2016-07-12-Hadoop-HDFS/">


<script class="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : false,
    isPost : true,
    lang   : 'zh'
  };
</script>
<title>Hadoop学习笔记(1)-HDFS | 玉丸子 | Blog</title>
  




  <noscript>
  <style>
  body { margin-top: 2rem; }

  .use-motion .menu-item,
  .use-motion .sidebar,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header {
    visibility: visible;
  }

  .use-motion .header,
  .use-motion .site-brand-container .toggle,
  .use-motion .footer { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle,
  .use-motion .custom-logo-image {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line {
    transform: scaleX(1);
  }

  .search-pop-overlay, .sidebar-nav { display: none; }
  .sidebar-panel { display: block; }
  </style>
</noscript>

<link rel="alternate" href="/yuwanzi.io/atom.xml" title="玉丸子 | Blog" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage" class="use-motion">
  <div class="headband"></div>

  <main class="main">
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="Navigationsleiste an/ausschalten" role="button">
    </div>
  </div>

  <div class="site-meta">

    <a href="/yuwanzi.io/" class="brand" rel="start">
      <i class="logo-line"></i>
      <h1 class="site-title">玉丸子 | Blog</h1>
      <i class="logo-line"></i>
    </a>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
    </div>
  </div>
</div>







</div>
        
  
  <div class="toggle sidebar-toggle" role="button">
    <span class="toggle-line"></span>
    <span class="toggle-line"></span>
    <span class="toggle-line"></span>
  </div>

  <aside class="sidebar">

    <div class="sidebar-inner sidebar-nav-active sidebar-toc-active">
      <ul class="sidebar-nav">
        <li class="sidebar-nav-toc">
          Inhaltsverzeichnis
        </li>
        <li class="sidebar-nav-overview">
          Übersicht
        </li>
      </ul>

      <div class="sidebar-panel-container">
        <!--noindex-->
        <div class="post-toc-wrap sidebar-panel">
            <div class="post-toc animated"><ol class="nav"><li class="nav-item nav-level-3"><a class="nav-link" href="#Hadoop%E6%A6%82%E8%BF%B0"><span class="nav-number">1.</span> <span class="nav-text">Hadoop概述</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#HDFS%E4%BD%93%E7%B3%BB%E7%BB%93%E6%9E%84"><span class="nav-number">2.</span> <span class="nav-text">HDFS体系结构</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#HDFSWriteOperation"><span class="nav-number">3.</span> <span class="nav-text">HDFSWriteOperation</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#HDFSReadOperation"><span class="nav-number">4.</span> <span class="nav-text">HDFSReadOperation</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#HDFSShell%E5%91%BD%E4%BB%A4"><span class="nav-number">5.</span> <span class="nav-text">HDFSShell命令</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E4%BD%BF%E7%94%A8JAVA%E6%93%8D%E4%BD%9CHDFS"><span class="nav-number">6.</span> <span class="nav-text">使用JAVA操作HDFS</span></a></li></ol></div>
        </div>
        <!--/noindex-->

        <div class="site-overview-wrap sidebar-panel">
          <div class="site-author site-overview-item animated" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <p class="site-author-name" itemprop="name">玉丸子</p>
  <div class="site-description" itemprop="description">这里是玉丸子的个人博客,与你一起发现更大的世界。</div>
</div>
<div class="site-state-wrap site-overview-item animated">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/yuwanzi.io/archives">
          <span class="site-state-item-count">68</span>
          <span class="site-state-item-name">Artikel</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/yuwanzi.io/categories/">
        <span class="site-state-item-count">39</span>
        <span class="site-state-item-name">Kategorien</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/yuwanzi.io/tags/">
        <span class="site-state-item-count">46</span>
        <span class="site-state-item-name">schlagwörter</span></a>
      </div>
  </nav>
</div>



        </div>
      </div>
    </div>
  </aside>
  <div class="sidebar-dimmer"></div>


    </header>

    
  <div class="back-to-top" role="button">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>

<noscript>
  <div class="noscript-warning">Theme NexT works best with JavaScript enabled</div>
</noscript>


    <div class="main-inner post posts-expand">


  


<div class="post-block">
  
  

  <article itemscope itemtype="http://schema.org/Article" class="post-content" lang="zh">
    <link itemprop="mainEntityOfPage" href="https://suyuhuan.gitee.io/yuwanzi.io/2016/07/12/2016-07-12-Hadoop-HDFS/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/yuwanzi.io/images/avatar.gif">
      <meta itemprop="name" content="玉丸子">
      <meta itemprop="description" content="这里是玉丸子的个人博客,与你一起发现更大的世界。">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="玉丸子 | Blog">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          Hadoop学习笔记(1)-HDFS
        </h1>

        <div class="post-meta-container">
          <div class="post-meta">
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar"></i>
      </span>
      <span class="post-meta-item-text">Veröffentlicht am</span>

      <time title="Erstellt: 2016-07-12 18:00:00" itemprop="dateCreated datePublished" datetime="2016-07-12T18:00:00+08:00">2016-07-12</time>
    </span>
      <span class="post-meta-item">
        <span class="post-meta-item-icon">
          <i class="far fa-calendar-check"></i>
        </span>
        <span class="post-meta-item-text">Bearbeitet am</span>
        <time title="Geändert am: 2020-11-07 08:58:17" itemprop="dateModified" datetime="2020-11-07T08:58:17+08:00">2020-11-07</time>
      </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-folder"></i>
      </span>
      <span class="post-meta-item-text">in</span>
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/yuwanzi.io/categories/%E5%90%8E%E7%AB%AF/" itemprop="url" rel="index"><span itemprop="name">后端</span></a>
        </span>
          . 
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/yuwanzi.io/categories/%E5%90%8E%E7%AB%AF/%E5%A4%A7%E6%95%B0%E6%8D%AE/" itemprop="url" rel="index"><span itemprop="name">大数据</span></a>
        </span>
          . 
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/yuwanzi.io/categories/%E5%90%8E%E7%AB%AF/%E5%A4%A7%E6%95%B0%E6%8D%AE/Hadoop/" itemprop="url" rel="index"><span itemprop="name">Hadoop</span></a>
        </span>
    </span>

  
</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">
        <p><img src="http://ww4.sinaimg.cn/mw690/63503acbjw1f5w446w8pcj20bp08rweu.jpg"></p>
<h3 id="Hadoop概述"><a href="#Hadoop概述" class="headerlink" title="Hadoop概述"></a>Hadoop概述</h3><p>Hadoop是一个由Apache基金会所开发的分布式系统基础架构。</p>
<p><img src="http://ww4.sinaimg.cn/mw690/63503acbjw1f5w4478njsj20as05wwep.jpg"></p>
<p>&nbsp;&nbsp;Hadoop 由许多元素构成。其最底部是 Hadoop Distributed File System（HDFS），它存储 Hadoop 集群中所有存储节点上的文件。HDFS的上一层是MapReduce 引擎，该引擎由 JobTrackers 和 TaskTrackers 组成。</p>
<p><strong>HDFS</strong></p>
<p>&nbsp;&nbsp;对外部客户机而言，HDFS就像一个传统的分级文件系统。可以创建、删除、移动或重命名文件，等等。但是 HDFS 的架构是基于一组特定的节点构建的，这是由它自身的特点决定的。这些节点包括 NameNode（仅一个），它在 HDFS 内部提供元数据服务；DataNode，它为 HDFS 提供存储块。由于仅存在一个 NameNode，因此这是 HDFS 的一个缺点（单点失败）。</p>
<p>&nbsp;&nbsp;存储在 HDFS 中的文件被分成块，然后将这些块复制到多个计算机中（DataNode）。这与传统的 RAID 架构大不相同。块的大小（通常为 64MB）和复制的块数量在创建文件时由客户机决定。NameNode 可以控制所有文件操作。HDFS 内部的所有通信都基于标准的 TCP/IP 协议。</p>
<p>&nbsp;&nbsp;HDFS和现有的分布式文件系统有很多共同点。但同时，它和其他的分布式文件系统的区别也是很明显的。HDFS是一个高度容错性的系统，适合部署在廉价的机器上。HDFS能提供高吞吐量的数据访问，非常适合大规模数据集上的应用。HDFS放宽了一部分POSIX约束，来实现流式读取文件系统数据的目的。HDFS在最开始是作为Apache Nutch搜索引擎项目的基础架构而开发的。HDFS是Apache Hadoop Core项目的一部分。</p>
<p><strong>NameNode</strong></p>
<p>&nbsp;&nbsp;NameNode 是一个通常在HDFS实例中的单独机器上运行的软件。它负责管理文件系统名称空间和控制外部客户机的访问。NameNode 决定是否将文件映射到 DataNode 上的复制块上。对于最常见的 3 个复制块，第一个复制块存储在同一机架的不同节点上，最后一个复制块存储在不同机架的某个节点上。</p>
<p>&nbsp;&nbsp;实际的 I/O事务并没有经过 NameNode，只有表示 DataNode 和块的文件映射的元数据经过 NameNode。当外部客户机发送请求要求创建文件时，NameNode 会以块标识和该块的第一个副本的 DataNode IP 地址作为响应。这个 NameNode 还会通知其他将要接收该块的副本的 DataNode。</p>
<p>&nbsp;&nbsp;NameNode 在一个称为FsImage的文件中存储所有关于文件系统名称空间的信息。这个文件和一个包含所有事务的记录文件（这里是 EditLog）将存储在 NameNode 的本地文件系统上。FsImage 和 EditLog 文件也需要复制副本，以防文件损坏或 NameNode 系统丢失。</p>
<p>&nbsp;&nbsp;NameNode本身不可避免地具有SPOF（Single Point Of Failure）单点失效的风险，主备模式并不能解决这个问题，通过Hadoop Non-stop namenode才能实现100% uptime可用时间。</p>
<p><strong>DataNode</strong></p>
<p>&nbsp;&nbsp;DataNode 也是一个通常在 HDFS实例中的单独机器上运行的软件。Hadoop 集群包含一个 NameNode 和大量 DataNode。DataNode通常以机架的形式组织，机架通过一个交换机将所有系统连接起来。Hadoop 的一个假设是：机架内部节点之间的传输速度快于机架间节点的传输速度。</p>
<p>&nbsp;&nbsp;DataNode 响应来自 HDFS 客户机的读写请求。它们还响应来自 NameNode 的创建、删除和复制块的命令。NameNode 依赖来自每个 DataNode 的定期心跳（heartbeat）消息。每条消息都包含一个块报告，NameNode 可以根据这个报告验证块映射和其他文件系统元数据。如果 DataNode 不能发送心跳消息，NameNode 将采取修复措施，重新复制在该节点上丢失的块。</p>
<h3 id="HDFS体系结构"><a href="#HDFS体系结构" class="headerlink" title="HDFS体系结构"></a>HDFS体系结构</h3><p><img src="http://ww2.sinaimg.cn/mw690/63503acbjw1f5w447vawzj20hq0aptar.jpg"></p>
<ul>
<li><strong>NameNode</strong>:唯一的master节点,管理HDFS的名称空间和数据块映射信息、配置副本策略和处理客户端请求。</li>
<li><strong>Secondary NameNode</strong>:辅助NameNode，分担NameNode工作，定期合并fsimage和edits并推送给NameNode，紧急情况下可辅助恢复NameNode。</li>
<li><strong>DataNode</strong>:Slave节点，实际存储数据、执行数据块的读写并汇报存储信息给NameNode。</li>
<li><strong>FSImage</strong>:元数据镜像文件。</li>
<li><strong>Edits</strong>:元数据的操作日志。</li>
</ul>
<h3 id="HDFSWriteOperation"><a href="#HDFSWriteOperation" class="headerlink" title="HDFSWriteOperation"></a>HDFSWriteOperation</h3><p><img src="http://ww3.sinaimg.cn/mw690/63503acbjw1f5w48p90chj20k20bzdgr.jpg"></p>
<p>&nbsp;&nbsp;在分布式文件系统中，需要确保数据的一致性。对于HDFS来说，直到所有要保存数据的DataNodes确认它们都有文件的副本时，数据才被认为写入完成。因此，数据一致性是在写的阶段完成的。一个客户端无论选择从哪个DataNode读取，都将得到相同的数据。</p>
<ol>
<li>客户端请求NameNode,表示写入文件。</li>
<li>NameNode响应客户端,并告诉客户端将文件保存到DataNodeA、B、D。</li>
<li>客户端连接DataNodeA写入文件,DataNode集群内完成复制。</li>
<li>DataNodeA将文件副本发送给DataNodeB。</li>
<li>DataNodeB将文件副本发送给DataNodeD。</li>
<li>DataNodeD返回确认消息给DataNodeB。</li>
<li>DataNodeB返回确认消息给DataNodeA。</li>
<li>DataNodeA返回确认消息给客户端,写入完成。</li>
</ol>
<p><img src="http://ww3.sinaimg.cn/mw690/63503acbjw1f5w448fsbnj20gn0a6acc.jpg"></p>
<ol>
<li>Client调用DistributedFileSystem的create()函数创建新文件。</li>
<li>DistributedFileSystem使用RPC调用NameNode创建一个没有block关联的新文件,NameNode在创建之前将进行校验,如果校验通过,NameNode则创建一个新文件并记录一条记录,否则抛出IO异常。</li>
<li>前两步成功后,将会返回一个DFSOutputStream对象,DFSOutputStream可以协调NameNode与DataNode,当客户端写入数据到DFSOutputStream,DFSOutputStream会将数据分割为一个一个Packet(数据包),并写入数据队列。</li>
<li>DataStreamer处理数据队列,它会先去询问NameNode存储到哪几个DataNode,例如Replication为3,则会去找到3个最适合的DataNode。DataStreamer会将DataNode排成一个Pipeline,它会将Packet按队列输出到管道中的第一个DataNode,第一个DataNode又会把Packet输出到第二个DataNode,直到最后一个DataNode。</li>
<li>DataStreamer中还有一个Ack Queue,Ack Queue之中也含有Packet。Ack Queue负责接收DataNode的确认响应,当Pipeline中的所有DataNode都确认完毕后,Ack Queue将移除对应的Packet。</li>
<li>Client完成数据写入,关闭流。</li>
<li>DataStreamer等待Ack Queue信息,当收到最后一个信息时,通知NameNode把文件标记为完成。</li>
</ol>
<h3 id="HDFSReadOperation"><a href="#HDFSReadOperation" class="headerlink" title="HDFSReadOperation"></a>HDFSReadOperation</h3><p><img src="http://ww4.sinaimg.cn/mw690/63503acbjw1f5w48ovukqj20jm0ermy3.jpg"></p>
<ol>
<li>客户端请求NameNode,表示读取文件。</li>
<li>NameNode响应客户端,将block(数据块)的信息发送给客户端。</li>
<li>客户端检查数据块信息,连接相关的DataNode。</li>
<li>DataNodeA将block1发送给客户端。</li>
<li>DataNodeB将block2发送给客户端。</li>
<li>拼接数据,读取完成。</li>
</ol>
<p><img src="http://ww2.sinaimg.cn/mw690/63503acbjw1f5w4486azhj20g209adi1.jpg"></p>
<ol>
<li>Client调用FileSystem的open()函数打开希望读取的文件。</li>
<li> DistributedFileSystem使用RPC调用NameNode确定文件起始块的位置，同一Block按照重复数会返回多个位置，这些位置按照Hadoop集群拓扑结构排序，距离客户端近的排在前面。</li>
<li>前两步成功后,将会返回一个DFSInputStream对象,DFSInputStream可以协调NameNode与DataNode。客户端对DFSInputStream输入流调用read()函数。</li>
<li>DFSInputStream连接距离最近的DataNode，通过对数据流反复调用read()函数，可以将数据从DataNode传输到客户端。</li>
<li>当到达Block的末端时，DFSInputStream会关闭与该DataNode的连接，然后寻找下一个Block的最佳DataNode，这些操作对客户端来说是透明的。</li>
<li>客户端完成读取，对FSDataInputStream调用close()关闭文件读取。</li>
</ol>
<h3 id="HDFSShell命令"><a href="#HDFSShell命令" class="headerlink" title="HDFSShell命令"></a>HDFSShell命令</h3><p>&nbsp;&nbsp;既然 HDFS 是存取数据的分布式文件系统，那么对 HDFS 的操作，就是文件系统的基本 操作，比如文件的创建、修改、删除、修改权限等，文件夹的创建、删除、重命名等。对 HDFS 的操作命令类似于 Linux 的 shell 对文件的操作，如 ls、mkdir、rm 等。 我们执行以下操作的时候，一定要确定 hadoop 是正常运行的，使用 jps 命令确保看到 各个 hadoop 进程。 </p>
<table>
<thead>
<tr>
<th>命令名</th>
<th>格式</th>
<th>含义</th>
</tr>
</thead>
<tbody><tr>
<td>-ls</td>
<td>-ls&lt;路径&gt;</td>
<td>查看指定路径的当前目录结构</td>
</tr>
<tr>
<td>-lsr</td>
<td>-lsr&lt;路径&gt;</td>
<td>递归查看指定路径的目录结构</td>
</tr>
<tr>
<td>-du</td>
<td>-du&lt;路径&gt;</td>
<td>统计目录下个文件大小</td>
</tr>
<tr>
<td>-dus</td>
<td>-dus&lt;路径&gt;</td>
<td>汇总统计目录下文件(夹)大小</td>
</tr>
<tr>
<td>-count</td>
<td>-count[-q]&lt;路径&gt;</td>
<td>统计文件(夹)数量</td>
</tr>
<tr>
<td>-mv</td>
<td>-mv&lt;源路径&gt;&lt;目的路径&gt;</td>
<td>移动</td>
</tr>
<tr>
<td>-cp</td>
<td>-cp&lt;源路径&gt;&lt;目的路径&gt;</td>
<td>复制</td>
</tr>
<tr>
<td>-rm</td>
<td>-rm[-skipTrash]&lt;路径&gt;</td>
<td>删除文件/空白文件夹</td>
</tr>
<tr>
<td>-rmr</td>
<td>-rmr[-skipTrash]&lt;路径&gt;</td>
<td>递归删除</td>
</tr>
<tr>
<td>-put</td>
<td>-put&lt;多个 linux 上的文件&gt;&lt;hdfs 路径&gt;</td>
<td>上传文件</td>
</tr>
<tr>
<td>-copyFromLocal</td>
<td>-copyFromLocal&lt;多个 linux 上的文件&gt; &lt;hdfs 路径&gt;</td>
<td>从本地复制</td>
</tr>
<tr>
<td>-moveFromLocal</td>
<td>-moveFromLocal&lt;多个 linux 上的文件&gt; &lt;hdfs 路径&gt;</td>
<td>从本地移动</td>
</tr>
<tr>
<td>-getmerge</td>
<td>-getmerge&lt;源路径&gt;&lt;linux 路径&gt;</td>
<td>合并到本地</td>
</tr>
<tr>
<td>-cat</td>
<td>-cat&lt;hdfs 路径&gt;</td>
<td>查看文件内容</td>
</tr>
<tr>
<td>-text</td>
<td>-text&lt;hdfs 路径&gt;</td>
<td>查看文件内容</td>
</tr>
<tr>
<td>-copyToLocal</td>
<td>-copyToLocal[-ignoreCrc][-crc][hdfs 源路 径][linux 目的路径]</td>
<td>复制到本地</td>
</tr>
<tr>
<td>-moveToLocal</td>
<td>-moveToLocal[-crc]&lt;hdfs 源路径&gt;&lt;linux 目的路径&gt;</td>
<td>移动到本地</td>
</tr>
<tr>
<td>-mkdir</td>
<td>-mkdir&lt;hdfs 路径&gt;</td>
<td>创建空白文件夹</td>
</tr>
<tr>
<td>-setrep</td>
<td>-setrep[-R][-w]&lt;副本数&gt;&lt;路径&gt;</td>
<td>修改副本数量</td>
</tr>
<tr>
<td>-touchz</td>
<td>-touchz&lt;文件路径&gt;</td>
<td>创建空白文件</td>
</tr>
<tr>
<td>-stat</td>
<td>-stat[format]&lt;路径&gt;</td>
<td>显示文件统计信息</td>
</tr>
<tr>
<td>-tail</td>
<td>-tail[-f]&lt;文件&gt;</td>
<td>查看文件尾部信息</td>
</tr>
<tr>
<td>-chmod</td>
<td>-chmod[-R]&lt;权限模式&gt;[路径]</td>
<td>修改权限</td>
</tr>
<tr>
<td>-chown</td>
<td>-chown[-R][属主][:[属组]] 路径</td>
<td>修改属主</td>
</tr>
<tr>
<td>-chgrp</td>
<td>-chgrp[-R] 属组名称 路径</td>
<td>修改属组</td>
</tr>
<tr>
<td>-help</td>
<td>-help[命令选项]</td>
<td>帮助</td>
</tr>
</tbody></table>
<h3 id="使用JAVA操作HDFS"><a href="#使用JAVA操作HDFS" class="headerlink" title="使用JAVA操作HDFS"></a>使用JAVA操作HDFS</h3><figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">HdfsTest</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">	<span class="keyword">private</span> Configuration conf = <span class="keyword">null</span>;</span><br><span class="line">	<span class="keyword">private</span> FileSystem fs = <span class="keyword">null</span>;</span><br><span class="line">	<span class="keyword">private</span> FSDataInputStream DFSInputStream = <span class="keyword">null</span>;</span><br><span class="line"></span><br><span class="line">	<span class="comment">/**</span></span><br><span class="line"><span class="comment">	 * 初始化FlieSystem</span></span><br><span class="line"><span class="comment">	 * </span></span><br><span class="line"><span class="comment">	 * <span class="doctag">@throws</span> IOException</span></span><br><span class="line"><span class="comment">	 * <span class="doctag">@throws</span> InterruptedException</span></span><br><span class="line"><span class="comment">	 */</span></span><br><span class="line">	<span class="meta">@Before</span></span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">init</span><span class="params">()</span> <span class="keyword">throws</span> IOException, InterruptedException </span>&#123;</span><br><span class="line">		conf = <span class="keyword">new</span> Configuration();</span><br><span class="line">		conf.set(<span class="string">&quot;fs.defaultFS&quot;</span>, <span class="string">&quot;hdfs://192.168.145.145:9000&quot;</span>);</span><br><span class="line">		fs = fs.get(URI.create(<span class="string">&quot;hdfs://192.168.145.145:9000&quot;</span>), conf, <span class="string">&quot;root&quot;</span>);</span><br><span class="line">	&#125;</span><br><span class="line"></span><br><span class="line">	<span class="comment">/**</span></span><br><span class="line"><span class="comment">	 * 读取文件</span></span><br><span class="line"><span class="comment">	 * </span></span><br><span class="line"><span class="comment">	 * <span class="doctag">@throws</span> IOException</span></span><br><span class="line"><span class="comment">	 * <span class="doctag">@throws</span> IllegalArgumentException</span></span><br><span class="line"><span class="comment">	 */</span></span><br><span class="line">	<span class="meta">@Test</span></span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">testReadAsOpen</span><span class="params">()</span> <span class="keyword">throws</span> IllegalArgumentException, IOException </span>&#123;</span><br><span class="line">		Path path = <span class="keyword">null</span>;</span><br><span class="line">		<span class="keyword">try</span> &#123;</span><br><span class="line">			path = <span class="keyword">new</span> Path(<span class="string">&quot;/test&quot;</span>);</span><br><span class="line">			<span class="keyword">if</span> (fs.exists(path)) &#123;</span><br><span class="line">				DFSInputStream = fs.open(path);</span><br><span class="line">				IOUtils.copyBytes(DFSInputStream, System.out, conf);</span><br><span class="line">			&#125;</span><br><span class="line">		&#125; <span class="keyword">finally</span> &#123;</span><br><span class="line">			IOUtils.closeStream(DFSInputStream);</span><br><span class="line">			fs.close();</span><br><span class="line">		&#125;</span><br><span class="line">	&#125;</span><br><span class="line"></span><br><span class="line">	<span class="comment">/**</span></span><br><span class="line"><span class="comment">	 * 上传本地文件</span></span><br><span class="line"><span class="comment">	 * </span></span><br><span class="line"><span class="comment">	 * <span class="doctag">@throws</span> IOException</span></span><br><span class="line"><span class="comment">	 */</span></span><br><span class="line">	<span class="meta">@Test</span></span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">testUpload</span><span class="params">()</span> <span class="keyword">throws</span> IOException </span>&#123;</span><br><span class="line">		Path src = <span class="keyword">null</span>;</span><br><span class="line">		Path dst = <span class="keyword">null</span>;</span><br><span class="line">		<span class="keyword">try</span> &#123;</span><br><span class="line">			src = <span class="keyword">new</span> Path(<span class="string">&quot;f:/saber_by_wlop-d8tjwa5.jpg&quot;</span>);<span class="comment">// 原路径</span></span><br><span class="line">			dst = <span class="keyword">new</span> Path(<span class="string">&quot;/saber.jpg&quot;</span>);<span class="comment">// 目标路径</span></span><br><span class="line">			<span class="comment">// 参数1为是否删除原文件,true为删除,默认为false</span></span><br><span class="line">			fs.copyFromLocalFile(<span class="keyword">false</span>, src, dst);</span><br><span class="line">			<span class="comment">// 打印文件路径</span></span><br><span class="line">			System.out.println(<span class="string">&quot;Upload to &quot;</span> + conf.get(<span class="string">&quot;fs.default.name&quot;</span>));</span><br><span class="line">			System.out.println(<span class="string">&quot;----------------------------------------&quot;</span>);</span><br><span class="line">			FileStatus[] fileStatus = fs.listStatus(dst);</span><br><span class="line">			<span class="keyword">for</span> (FileStatus file : fileStatus) &#123;</span><br><span class="line">				System.out.println(file.getPath());</span><br><span class="line">			&#125;</span><br><span class="line">		&#125; <span class="keyword">finally</span> &#123;</span><br><span class="line">			fs.close();</span><br><span class="line">		&#125;</span><br><span class="line">	&#125;</span><br><span class="line"></span><br><span class="line">	<span class="comment">/**</span></span><br><span class="line"><span class="comment">	 * 下载文件</span></span><br><span class="line"><span class="comment">	 * </span></span><br><span class="line"><span class="comment">	 * <span class="doctag">@throws</span> IOException</span></span><br><span class="line"><span class="comment">	 */</span></span><br><span class="line">	<span class="meta">@Test</span></span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">testDownload</span><span class="params">()</span> <span class="keyword">throws</span> IOException </span>&#123;</span><br><span class="line">		Path src = <span class="keyword">null</span>;</span><br><span class="line">		Path dst = <span class="keyword">null</span>;</span><br><span class="line">		<span class="keyword">try</span> &#123;</span><br><span class="line">			<span class="keyword">if</span> (fs.exists(src)) &#123;</span><br><span class="line">				src = <span class="keyword">new</span> Path(<span class="string">&quot;/saber.jpg&quot;</span>);</span><br><span class="line">				dst = <span class="keyword">new</span> Path(<span class="string">&quot;D:/temp/&quot;</span>);</span><br><span class="line">				fs.copyToLocalFile(<span class="keyword">false</span>, src, dst);</span><br><span class="line">				<span class="comment">// 打印文件路径</span></span><br><span class="line">				System.out.println(<span class="string">&quot;Download from &quot;</span> + conf.get(<span class="string">&quot;fs.default.name&quot;</span>));</span><br><span class="line">				System.out.println(<span class="string">&quot;--------------------------------------------&quot;</span>);</span><br><span class="line">				<span class="comment">// 迭代路径,参数2为是否递归迭代</span></span><br><span class="line">				RemoteIterator&lt;LocatedFileStatus&gt; iterator = fs.listFiles(src, <span class="keyword">true</span>);</span><br><span class="line">				<span class="keyword">while</span> (iterator.hasNext()) &#123;</span><br><span class="line">					LocatedFileStatus fileStatus = iterator.next();</span><br><span class="line">					System.out.println(fileStatus.getPath());</span><br><span class="line">				&#125;</span><br><span class="line">			&#125;</span><br><span class="line">		&#125; <span class="keyword">finally</span> &#123;</span><br><span class="line">			fs.close();</span><br><span class="line">		&#125;</span><br><span class="line">	&#125;</span><br><span class="line"></span><br><span class="line">	<span class="comment">/**</span></span><br><span class="line"><span class="comment">	 * 创建目录</span></span><br><span class="line"><span class="comment">	 * </span></span><br><span class="line"><span class="comment">	 * <span class="doctag">@throws</span> IOException</span></span><br><span class="line"><span class="comment">	 */</span></span><br><span class="line">	<span class="meta">@Test</span></span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">testMkdir</span><span class="params">()</span> <span class="keyword">throws</span> IOException </span>&#123;</span><br><span class="line">		Path path = <span class="keyword">null</span>;</span><br><span class="line">		<span class="keyword">try</span> &#123;</span><br><span class="line">			path = <span class="keyword">new</span> Path(<span class="string">&quot;/create01&quot;</span>);</span><br><span class="line">			<span class="comment">// 判断目录是否已存在</span></span><br><span class="line">			<span class="keyword">boolean</span> exists = fs.exists(path);</span><br><span class="line">			<span class="keyword">if</span> (!exists) &#123;</span><br><span class="line">				<span class="comment">// 创建目录</span></span><br><span class="line">				<span class="keyword">boolean</span> mkdirs = fs.mkdirs(path);</span><br><span class="line">				<span class="keyword">if</span> (mkdirs) &#123;</span><br><span class="line">					System.out.println(<span class="string">&quot;create dir success!&quot;</span>);</span><br><span class="line">				&#125; <span class="keyword">else</span> &#123;</span><br><span class="line">					System.out.println(<span class="string">&quot;create dir failure!&quot;</span>);</span><br><span class="line">				&#125;</span><br><span class="line">			&#125;</span><br><span class="line">		&#125; <span class="keyword">finally</span> &#123;</span><br><span class="line">			fs.close();</span><br><span class="line">		&#125;</span><br><span class="line">	&#125;</span><br><span class="line"></span><br><span class="line">	<span class="comment">/**</span></span><br><span class="line"><span class="comment">	 * 重命名文件</span></span><br><span class="line"><span class="comment">	 * </span></span><br><span class="line"><span class="comment">	 * <span class="doctag">@throws</span> IOException</span></span><br><span class="line"><span class="comment">	 */</span></span><br><span class="line">	<span class="meta">@Test</span></span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">testRename</span><span class="params">()</span> <span class="keyword">throws</span> IOException </span>&#123;</span><br><span class="line">		Path oldPath = <span class="keyword">null</span>;</span><br><span class="line">		Path newPath = <span class="keyword">null</span>;</span><br><span class="line">		<span class="keyword">try</span> &#123;</span><br><span class="line">			oldPath = <span class="keyword">new</span> Path(<span class="string">&quot;/saber.jpg&quot;</span>);</span><br><span class="line">			newPath = <span class="keyword">new</span> Path(<span class="string">&quot;/saber01.jpg&quot;</span>);</span><br><span class="line">			<span class="keyword">if</span> (fs.exists(oldPath)) &#123;</span><br><span class="line">				<span class="keyword">boolean</span> rename = fs.rename(oldPath, newPath);</span><br><span class="line">				<span class="keyword">if</span> (rename) &#123;</span><br><span class="line">					System.out.println(<span class="string">&quot;rename success!&quot;</span>);</span><br><span class="line">				&#125; <span class="keyword">else</span> &#123;</span><br><span class="line">					System.out.println(<span class="string">&quot;rename failure!&quot;</span>);</span><br><span class="line">				&#125;</span><br><span class="line">			&#125;</span><br><span class="line">		&#125; <span class="keyword">finally</span> &#123;</span><br><span class="line">			fs.close();</span><br><span class="line">		&#125;</span><br><span class="line">	&#125;</span><br><span class="line"></span><br><span class="line">	<span class="comment">/**</span></span><br><span class="line"><span class="comment">	 * 删除文件</span></span><br><span class="line"><span class="comment">	 * </span></span><br><span class="line"><span class="comment">	 * <span class="doctag">@throws</span> IOException</span></span><br><span class="line"><span class="comment">	 */</span></span><br><span class="line">	<span class="meta">@Test</span></span><br><span class="line">	<span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">testDelete</span><span class="params">()</span> <span class="keyword">throws</span> IOException </span>&#123;</span><br><span class="line">		Path path = <span class="keyword">null</span>;</span><br><span class="line">		<span class="keyword">try</span> &#123;</span><br><span class="line">			path = <span class="keyword">new</span> Path(<span class="string">&quot;/saber01.jpg&quot;</span>);</span><br><span class="line">			<span class="keyword">if</span> (fs.exists(path)) &#123;</span><br><span class="line">				<span class="keyword">boolean</span> delete = fs.deleteOnExit(path);</span><br><span class="line">				<span class="keyword">if</span> (delete) &#123;</span><br><span class="line">					System.out.println(<span class="string">&quot;delete success!&quot;</span>);</span><br><span class="line">				&#125; <span class="keyword">else</span> &#123;</span><br><span class="line">					System.out.println(<span class="string">&quot;delete failure!&quot;</span>);</span><br><span class="line">				&#125;</span><br><span class="line">			&#125;</span><br><span class="line">		&#125; <span class="keyword">finally</span> &#123;</span><br><span class="line">			fs.close();</span><br><span class="line">		&#125;</span><br><span class="line">	&#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
    </div>

    
    
    

    <footer class="post-footer">
          <div class="post-tags">
              <a href="/yuwanzi.io/tags/Hadoop/" rel="tag"># Hadoop</a>
              <a href="/yuwanzi.io/tags/%E5%A4%A7%E6%95%B0%E6%8D%AE/" rel="tag"># 大数据</a>
          </div>

        

          <div class="post-nav">
            <div class="post-nav-item">
                <a href="/yuwanzi.io/2016/07/09/2016-07-09-MyCat/" rel="prev" title="MyCat快速入门">
                  <i class="fa fa-chevron-left"></i> MyCat快速入门
                </a>
            </div>
            <div class="post-nav-item">
                <a href="/yuwanzi.io/2016/07/14/2016-07-14-Hadoop02-MapReduce/" rel="next" title="Hadoop学习笔记(2)-Mapreduce">
                  Hadoop学习笔记(2)-Mapreduce <i class="fa fa-chevron-right"></i>
                </a>
            </div>
          </div>
    </footer>
  </article>
</div>







<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      const activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      const commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>
</div>
  </main>

  <footer class="footer">
    <div class="footer-inner">


<div class="copyright">
  &copy; 
  <span itemprop="copyrightYear">2021</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">玉丸子</span>
</div>
  <div class="powered-by">Erstellt mit  <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://theme-next.js.org/muse/" class="theme-link" rel="noopener" target="_blank">NexT.Muse</a>
  </div>

    </div>
  </footer>

  
  <script src="//cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
<script src="/yuwanzi.io/js/utils.js"></script><script src="/yuwanzi.io/js/motion.js"></script><script src="/yuwanzi.io/js/schemes/muse.js"></script><script src="/yuwanzi.io/js/next-boot.js"></script>

  






  





</body>
</html>
